package jianger.v1;

import com.google.gson.Gson;
import com.google.gson.internal.LinkedTreeMap;
import jianger.pojo.Product;
import jianger.utils.GetDocumentUtils;
import jianger.utils.ProductDao;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashSet;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * 爬取京东手机的第三版代码
 * 使用线程池+阻塞队列
 *
 * @author jianger
 * @version v2.0
 * @ date 2018/2/25
 */
public class SpiderV4 {

    /**
     * 使用多线程分别爬取网站，存储html，获取pid，存到数据库
     * 1. 定义3个阻塞队列，htmlQueue,pidQueue,productQueue
     * 2. 使用htmlQueue存储html。
     */

    private static ProductDao productDao = new ProductDao();
    private static ExecutorService threadPool = Executors.newFixedThreadPool(52);


    private static ArrayBlockingQueue<Document> htmlQueue = new ArrayBlockingQueue<Document>(100);
    private static ArrayBlockingQueue<String> pidQueue = new ArrayBlockingQueue<String>(1000);
    private static ArrayBlockingQueue<Product> productQueue = new ArrayBlockingQueue<Product>(100);


    public static void main(String[] args) throws IOException {

        //首页的url  https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&&pvid=30fdd1f5980042afa987d10636aae503


        // 开启四个线程，对每个队列中的数据进行监控

            threadPool.execute(new Runnable() {
                public void run() {

                    while (true){

                        try {
                            Thread.sleep(2000);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }

                        System.out.println("html中的数量为----"+htmlQueue.size());
                        System.out.println("pid中的数量为------"+pidQueue.size());
                        System.out.println("product中的数量为------"+productQueue.size());
                    }


                }
            });


        // 开启四个线程解析document,消耗htmlQueue.获取pid，把pid存入pidQueue

        for (int i = 0; i <= 4; i++) {
            threadPool.execute(new Runnable() {
                public void run() {
                    while (true) {
                        Document document = null;
                        try {
                            document = htmlQueue.take();
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                        assert document != null;
                        Elements elements = document.select("[data-pid]");
                        HashSet<String> pids = new HashSet<String>();
                        if (elements.size() != 0) {
                            for (Element element : elements) {
                                String pid = element.attr("data-pid"); // 获取每个手机的pid
                                pids.add(pid);
                            }
                        }
                        for (final String pid : pids) {

                            try {
                                pidQueue.put(pid);
                            } catch (InterruptedException e) {
                                try {
                                    htmlQueue.put(document);
                                } catch (InterruptedException e1) {
                                    e1.printStackTrace();
                                }
                            }

                        }

                    }

                }
            });
        }


        // 开启四个线程。消耗pid，产生product，存入productQueue

        for (int i = 0; i <= 30; i++) {
            threadPool.execute(new Runnable() {
                public void run() {
                    while (true) {
                        String pid = null;
                        try {
                            pid = pidQueue.take();

                            // 详情页的url  https://item.jd.com/5025971.html
                            String detailUrl = "https://item.jd.com/" + pid + ".html"; // 拼接手机详情页的url

                            // 访问每个手机的详情页
                            Document dateil = null;
                            dateil = GetDocumentUtils.getDecument(detailUrl);

                            String phoneName = dateil.select(".sku-name").text(); //解析出手机的名字
                            //因为价格是ajax异步加载的，所以要获取价格需要一部的解析价格的url

                            String pirceUrl = "https://p.3.cn/prices/mgets?pduid=2018573997&skuIds=J_" + pid;
                            // 1. 获取一个httpGet对象
                            HttpGet httpGet = new HttpGet(pirceUrl);
                            // 2.获取一个httpclient对象，用来执行http请求
                            CloseableHttpClient httpclient = HttpClients.createDefault();
                            // 3.执行请求，并得到返回对象
                            CloseableHttpResponse response = httpclient.execute(httpGet);

                            // 4. 返回一个html
                            String html = EntityUtils.toString(response.getEntity(), Charset.forName("utf-8"));
                            Gson gson = new Gson();
                            @SuppressWarnings("unchecked")
                            List<LinkedTreeMap<String, String>> list = (List<LinkedTreeMap<String, String>>) gson.fromJson(html, List.class);
                            //获取list第一个元素
                            LinkedTreeMap<String, String> map = list.get(0);
                            String price = map.get("p");
                            //每次循环都创建一个product对象
                            Product product = new Product(pid, phoneName, detailUrl, price);
                            productQueue.put(product);
                        } catch (Exception e) {
                                e.printStackTrace();
                        }


                    }


                }
            });

        }

        // 开启四个线程，从productQueue中取出product，存在数据库

        for (int i = 0; i <= 14; i++)
            threadPool.execute(new Runnable() {
                public void run() {
                    Product product = null;
                    while (true) {

                        try {
                            product = productQueue.take();
                            productDao.save(product);

                        } catch (InterruptedException e) {

                            try {
                                assert product != null;
                                productQueue.put(product);
                            } catch (InterruptedException e1) {
                                e1.printStackTrace();
                            }
                        }
                    }


                }
            });


        indexPhone();
        //2. 翻页
        doPaging();
    }

    private static void doPaging() throws IOException {
        // 第二部分：解析搜索结果的翻页数据
        // 分页请求
        // 分页接口：https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&cid2=653&cid3=655&page=3&s=60&click=0
        // 能够变化两个字：page，s 。s推测是时间戳，并 且测试之后可以删除掉。
        // page 的值 page=2n-1 n是第几页
        // 总共分100页
        int totalPage = 100;
        int page = 0;
        // 从第二页开始执行分页
        for (int i = 2; i <= totalPage; i++) {
            page = i * 2 - 1;
            String pagingUrl = "https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&cid2=653&cid3=655&page=" + page + "&click=0";
            //发送请求
            Document document = GetDocumentUtils.getDecument(pagingUrl);
            try {
                htmlQueue.put(document);
            } catch (InterruptedException e) {
                System.out.println("翻页失败" + page);
            }

        }
    }


    //解析首页，返回pids
    private static void indexPhone() throws IOException {
        // 1. 解析首页，传入关键字手机
        String keyWorld = "手机";
        String indexUrl = "https://search.jd.com/Search?keyword=" + keyWorld
                + "&enc=utf-8&pvid=9f8a79872dc644699e6a62597d745ce4";
        Document document = GetDocumentUtils.getDecument(indexUrl); // 获取手机首页的document
        try {
            htmlQueue.put(document);
        } catch (InterruptedException e) {
            System.out.println("首页解析失败");
        }

    }


}

